package myframe.htmlutil;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Scanner;
import java.util.Set;

import org.apache.http.impl.client.HttpClientBuilder;

import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.toolkit.StringUtils;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.CookieManager;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlButton;
import com.gargoylesoftware.htmlunit.html.HtmlImage;
import com.gargoylesoftware.htmlunit.html.HtmlInput;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.Cookie;

import lombok.extern.slf4j.Slf4j;


@Slf4j
public class HtmlDo {
	
	static Set<String> urlSet = new HashSet<String>();
	
	public static void main(String[] args) throws Exception {  
		 //HtmlPage page = HtmlDo.pceggs();
		 //HtmlDo.pceggs();
		//testgg(cookies_ret);
		 //   HtmlDo.testgg();
		//  String asXml = page.asXml();
		//  System.out.println(asXml);
		 HtmlDo.testgg(null, "https://login.taobao.com/member/loginByIm.do?uid=cntaobaocowbin&token=6cb3a919f550348047a12b3e9ce4dc78&time=1541124359198&asker=qrcodelogin&ask_version=1.0.0&defaulturl=http%3A%2F%2Fcart.taobao.com%2Fcart.htm%3Fspm%3Da1z09.2.a2109.d1000367.418e2e8dRB0seU%26nekot%3D1470211439694&webpas=eb77a2ecff2bc370a7a7cffdefc3ea891354527844&umid_token=C1541121604947628248915681541124144070881");
		// https://login.taobao.com/member/loginByIm.do?uid=cntaobaocowbin&token=6cb3a919f550348047a12b3e9ce4dc78&time=1541124359198&asker=qrcodelogin&ask_version=1.0.0&defaulturl=http%3A%2F%2Fcart.taobao.com%2Fcart.htm%3Fspm%3Da1z09.2.a2109.d1000367.418e2e8dRB0seU%26nekot%3D1470211439694&webpas=eb77a2ecff2bc370a7a7cffdefc3ea891354527844&umid_token=C1541121604947628248915681541124144070881
	}
	
	
	public static void testgg(Set<Cookie> cookies_ret,String url) throws FailingHttpStatusCodeException, IOException{
		  // String  url=url;//想采集的网址
         //  String refer="http://outofmemory.cn/";
        //   URL link=new URL(url); 
           WebClient wc=new WebClient(BrowserVersion.CHROME);;
          // WebRequest request=new WebRequest(link); 
 
	      // request.setProxyHost("120.120.120.78");
          // request.setProxyPort(8080);
         //  request.setAdditionalHeader("Referer", refer);//设置请求报文头里的refer字段
           ////设置请求报文头里的User-Agent字段
          // request.setAdditionalHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
           //wc.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
           //wc.addRequestHeader和request.setAdditionalHeader功能应该是一样的。选择一个即可。
           //其他报文头字段可以根据需要添加
          
           wc.getCookieManager().setCookiesEnabled(true);//开启cookie管理
           wc.getOptions().setJavaScriptEnabled(true);//开启js解析。对于变态网页，这个是必须的
           wc.getOptions().setCssEnabled(true);//开启css解析。对于变态网页，这个是必须的。
           wc.getOptions().setThrowExceptionOnFailingStatusCode(false);
           wc.getOptions().setThrowExceptionOnScriptError(false);
           wc.getOptions().setTimeout(1000);
           //设置cookie。如果你有cookie，可以在这里设置
           Set<Cookie> cookies=cookies_ret;
           if(cookies != null){
        	   Iterator<Cookie> i = cookies.iterator();
               while (i.hasNext()) 
               {
                   wc.getCookieManager().addCookie(i.next());
               } 
           }
           
           //准备工作已经做好了
           //HtmlPage page=null;           
          // page = wc.getPage(request);
           HtmlPage page = wc.getPage(url);
           CookieManager CM = wc.getCookieManager();
           if( CM.getCookies() != null){
        	   cookies_ret = CM.getCookies();
           }  
          
           String content=page.asXml();
           
           url = url.replace("http://", "");
		   url = url.replace("https://", "");
		   url = "D:/"+url;
           int lastIndex = url.lastIndexOf("/");
	       String path = url.substring(0,lastIndex);
	       File pathFile = new File(path);
	       if(!pathFile.exists()){
	        	pathFile.mkdirs();
	        }
	       url = url + ".html";
	       
	       File ftest = new File(url);
           page.save(ftest);
	       
	       
	       
	       /*    FileOutputStream fos = new FileOutputStream(url);
           String s = content;
           String str1=url.replace("/","");
           int count = url.length()-str1.length();
           String ss = "";
           for(int j = 0 ;j<count-2;j++){
        	   ss = ss + "../";
           }
           if(s!=null){
        	   s=s.replace("http://cdn.admui.com/demo/", ss);
        	   
           }
           fos.write(s.getBytes());
           fos.close();
           
           downLoadJsCss(page);*/
           
          
          // String content=page.asText();//网页内容保存在content里
         //  log.info(content);
           /*if(page==null)
           {
               System.out.println("采集 "+url+" 失败!!!");
               return ;
           }
           String content=page.asText();//网页内容保存在content里
           if(content==null)
           {
               System.out.println("采集 "+url+" 失败!!!");
               return ;
           }
           //搞定了
           CookieManager CM = wc.getCookieManager(); //WC = Your WebClient's name
           Set<Cookie> cookies_ret = CM.getCookies();
           log.info(JSON.toJSONString(cookies_ret));*/
	}
	
	
	  //某网站  需要验证码
	  public static HtmlPage pceggs() throws Exception{
	        //设置浏览器内核
	         WebClient webClient = new WebClient(BrowserVersion.CHROME);
	         webClient.getOptions().setThrowExceptionOnScriptError(false);
	         webClient.getOptions().setTimeout(1000);
	         System.out.println(HttpClientBuilder.class.getProtectionDomain().getCodeSource().getLocation());
	         //设置证书
	       //  webClient.getOptions().setUseInsecureSSL(true);
	         //是否启用js
	        // webClient.getOptions().setJavaScriptEnabled(true);
	         //是否启用css
	        // webClient.getOptions().setCssEnabled(true);
	         
	         webClient.getCookieManager().setCookiesEnabled(true);
	         //设置Ajax控制器
	         webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	         //获取页面  
	         HtmlPage page = webClient.getPage("https://login.tmall.com/");//登陆页面
	         System.out.println(page.asText());
	         HtmlInput username = (HtmlInput) page.getElementById("J_Quick2Static");//账号输入框的id
	         HtmlInput password = (HtmlInput) page.getElementById("password");//密码输入框的id
	     //    HtmlInput code=(HtmlInput) page.getElementById("txt_VerifyCode");//验证码输入栏的id
	         HtmlInput code=(HtmlInput) page.getElementByName("validCode");
	        // HtmlInput login=(HtmlInput) page.getElementById("Login_Submit");
	         DomNodeList<DomElement> buttonList =  page.getElementsByTagName("button");
	         HtmlButton login= null;
	         for(DomElement dom:buttonList){
	        	 System.out.println(dom.getAttribute("class"));
	        	 if(dom.getAttribute("class")!=null && dom.getAttribute("class").equals("btn btn-primary btn-block margin-top-30")){
	        		 login = (HtmlButton)(dom);
	        	 }
	         }
	        HtmlImage vaCode=  null;
	        DomNodeList<DomElement> aList =  page.getElementsByTagName("img");
	        for(DomElement dom:aList){
	        	System.out.println(dom.getAttribute("src"));
	        	if(dom.getAttribute("src")!=null && dom.getAttribute("src").equals("/system/captcha")){
	        		vaCode = (HtmlImage)(dom);
	        	 }
	        }
	        //验证码图片存储 参数：验证码的图片id
	       // HtmlImage vaCode=(HtmlImage) page.getElementById("valiCode");
	        File file=new File("yzm.png");
	        //保存图片到项目根目录下
	        vaCode.saveAs(file);
	        Scanner scanner=new Scanner(System.in);
	        //查看你项目下的yzm.png 手动输入
	        String codes=scanner.nextLine();
	        username.setAttribute("value","13422637324");;
	        password.setAttribute("value","aa123456");
	        code.setAttribute("value", codes);
	        page=login.click();
	        
	        String content =  page.asXml();
	        content.replace("http://cdn.admui.com/demo/", "");
	        
	        CookieManager CM = webClient.getCookieManager(); //WC = Your WebClient's name
	        Set<Cookie> cookies_ret = CM.getCookies();
	        log.info(JSON.toJSONString(cookies_ret));
	   //     return cookies_ret;
	        
	        // page 就是返回结果，如果需要html形式 page.asXml 如果是需要Text 则使用 page.asText
	        
	      //  downLoadJsCss(page);
	        
	        DomNodeList<DomElement> aliList =  page.getElementsByTagName("a");
	        for(DomElement dom:aliList){
	        	try{
	        		
	        		if(StringUtils.isNotEmpty(dom.getAttribute("href")) && dom.getAttribute("href").indexOf("http")<0
	        				&&  dom.getAttribute("href").indexOf("logout")== -1){
	        			//HtmlPage page2 = dom.click();	        			
	        			String url = dom.getAttribute("href");
		        		url = url.replace("/../", "/");
	        			url = "http://iframe.demo.admui.com"+url;
	        			url = url.replace("#", "/");
	        			//new T(url, cookies_ret).start();	  
	        			HtmlDo.testgg( cookies_ret, url);
	        			if(dom.getAttribute("href").contains("/components/")
	        					||dom.getAttribute("href").contains("/charts/")
	        					||dom.getAttribute("href").contains("/forms/")
	        					||dom.getAttribute("href").contains("/pages/")
	        					||dom.getAttribute("href").contains("/system/")
	        					||dom.getAttribute("href").contains("/tables/")){
	        				String replaceUrl = "admui"+dom.getAttribute("href")+".html";
	        				content = content.replace("=\""+dom.getAttribute("href"),"=\""+replaceUrl);
	        			}
	        		}
	        		
	        	}catch(Exception e){
	        		
	        	}
	        }
	        
	        
           FileOutputStream fos = new FileOutputStream("D:/cdn.admui.com/index.html");
           String s = content;
           fos.write(s.getBytes());
           fos.close();
	        
	        
	        System.out.println("end................");
	        System.out.println("end................");
	        System.out.println("end................");
	        return page;
	  }
	  
	  //无需验证码
	  public static HtmlPage lezhuan1() throws Exception {
	  //设置浏览器内核
	         WebClient webClient = new WebClient(BrowserVersion.CHROME);
	         //设置证书
	         webClient.getOptions().setUseInsecureSSL(true);
	         //是否启用js
	         webClient.getOptions().setJavaScriptEnabled(true);
	         //是否启用css
	         webClient.getOptions().setCssEnabled(true);
	         //设置Ajax控制器
	         webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	         //获取页面  
	         HtmlPage page = webClient.getPage("http://www.lezhuan1.com/login.html");
	         HtmlInput username = (HtmlInput) page.getElementById("tbUserAccount");
	         HtmlInput password = (HtmlInput) page.getElementById("tbUserPwd");
	         HtmlAnchor btn=page.getFirstByXPath("//li/a[@class='LoginBtn']");
	         username.setAttribute("value","你的账号");;
	         password.setAttribute("value","密码");
	         //这个网站比较的特殊它的登陆按钮是一张图片，所以需要找到它的li 标签下的a 标签  class为LoginBtn识别
	         page=btn.click();
	        return page;
	  }
	  
	  
	  public static void downLoadJsCss(HtmlPage page){
		  DomNodeList<DomElement> scriptList =  page.getElementsByTagName("script");
	        for(DomElement dom:scriptList){
	        	try{
	        		String url = dom.getAttribute("src");
	        		url = url.replace("/../", "/");
	        		if(StringUtils.isNotEmpty(url) && url.indexOf("http")>=0){
	        			String path = url.replace("http://", "");
	        			path = path.replace("https://", "");
	        			path = "D:/"+path;
	        			downloadFile(url,path);
		        		log.info(url);
	        		}
	        		
	        	}catch(Exception e){
	        		
	        	}
	        	
	        }
	        
	        DomNodeList<DomElement> linkList =  page.getElementsByTagName("link");
	        for(DomElement dom:linkList){
	        	try{
	        		String url = dom.getAttribute("href");
	        		url = url.replace("/../", "/");
	        		if(StringUtils.isNotEmpty(url) && url.indexOf("http")>=0){
	        			String path = url.replace("http://", "");
	        			path = path.replace("https://", "");
	        			path = "D:/"+path;
	        			downloadFile(url,path);
		        		log.info(url);
	        		}
	        		
	        	}catch(Exception e){
	        		
	        	}
	        }
	        
	        
	        DomNodeList<DomElement> imgList =  page.getElementsByTagName("img");
	        for(DomElement dom:imgList){
	        	try{
	        		String url = dom.getAttribute("src");
	        		url = url.replace("/../", "/");
	        		if(StringUtils.isNotEmpty(url) && url.indexOf("http")>=0){
	        			String path = url.replace("http://", "");
	        			path = path.replace("https://", "");
	        			path = "D:/"+path;
	        			downloadFile(url,path);
		        		log.info(url);
	        		}
	        		
	        	}catch(Exception e){
	        		
	        	}
	        	
	        }
	        
	        
	        DomNodeList<DomElement> metaList =  page.getElementsByTagName("meta");
	        for(DomElement dom:metaList){
	        	try{
	        		String url = dom.getAttribute("href");
	        		url = url.replace("/../", "/");
	        		if(StringUtils.isNotEmpty(url) && url.indexOf("http")>=0){
	        			String path = url.replace("http://", "");
	        			path = path.replace("https://", "");
	        			path = "D:/"+path;
	        			downloadFile(url,path);
		        		log.info(url);
	        		}
	        		
	        	}catch(Exception e){
	        		
	        	}
	        	
	        }
	  }
	  
	  public static void downloadFile(String remoteFilePath, String localFilePath){
		   if(!urlSet.contains(remoteFilePath)){
			   urlSet.add(remoteFilePath);
			   URL urlfile = null;
		        HttpURLConnection httpUrl = null;
		        BufferedInputStream bis = null;
		        BufferedOutputStream bos = null;
		        if(remoteFilePath.indexOf("?")>=0){
		        	int urlindex = remoteFilePath.lastIndexOf("?");
			        remoteFilePath = remoteFilePath.substring(0,urlindex);
		        }
		        int lastIndex = localFilePath.lastIndexOf("/");
		        String path = localFilePath.substring(0,lastIndex);
		        File pathFile = new File(path);
		        if(!pathFile.exists()){
		        	pathFile.mkdirs();
		        }
		        File f = new File(localFilePath);
		        try
		        {
		            urlfile = new URL(remoteFilePath);
		            httpUrl = (HttpURLConnection)urlfile.openConnection();
		            httpUrl.connect();
		            bis = new BufferedInputStream(httpUrl.getInputStream());
		            bos = new BufferedOutputStream(new FileOutputStream(f));
		            int len = 2048;
		            byte[] b = new byte[len];
		            while ((len = bis.read(b)) != -1)
		            {
		                bos.write(b, 0, len);
		            }
		            System.out.println("上传成功");
		            bos.flush();
		            bis.close();
		            httpUrl.disconnect();
		        }
		        catch (Exception e)
		        {
		            //e.printStackTrace();
		        }
		        finally
		        {
		            try
		            {
		                bis.close();
		                bos.close();
		            }
		            catch (IOException e)
		            {
		              //  e.printStackTrace();
		            }
		        }
		   }
	        
	    } 
	  
}

class T extends Thread{
	
	
	public String url;
	
	public Set<Cookie> cookies_ret;
	
	public T(String url,Set<Cookie> cookies_ret){
		this.url = url;
		this.cookies_ret = cookies_ret;
	}
	
	public void run(){
		try {
			HtmlDo.testgg( cookies_ret, url);
		} catch (FailingHttpStatusCodeException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
}
